import requests  # 爬取晋江小说网
import pymysql
import time  # 获取时间
import re  # 正则表达式
import os  # 系统相关操作
from bs4 import BeautifulSoup  # bs4网页解析


class mysql:
    db = pymysql.connect

    def __init__(self, url, username, pwd, dbname):
        self.db = pymysql.connect(url, username, pwd, dbname)

    def __del__(self):
        self.db.close()

    def do(self, sql):
        try:
            cursor = self.db.cursor()
            cursor.execute(sql)     # 执行sql语句
            self.db.commit()        # 执行sql语句
            print("=====已插入====")
            time.sleep(1)
            return cursor.fetchall()
        except Exception:
            print(Exception)


if __name__ == '__main__':
    a = mysql("47.94.92.138", "cq", "123456", "novel")
    for i in range(0, 5):

        yuanBookId = 719747 + i
        strNum = "第 %d 本：%d" % (i+1, yuanBookId)
        print(strNum)

        url = 'http://www.jjwxc.net/onebook.php?novelid={}'.format(yuanBookId)
        # http://www.jjwxc.net/onebook.php?novelid=1248812 游医

        res = requests.get(url)  # 获取网页内容
        res.encoding = 'gb18030'  # 转成该网址的格式

        soup = BeautifulSoup(res.text, 'lxml')  # 解析网页脚本

        # =================基础判断==========================
        # 没有作品页面跳过
        title = soup.find_all('title')
        if title and title[0].text == "未发文作者的作品页":
            print("本ID没有文章，跳过\n")
            continue

        # 作者写了但是未公布跳过
        pattern_pass = re.compile(r'该文全文存稿中，作者大大尚未打算曝光，暂时还不能看哦~', re.S)
        needPass = pattern_pass.findall(res.text)
        if needPass:
            print("该作品未发布，跳过\n")
            continue

        # 小说字数（为0跳过
        novelWords = soup.find_all('span', attrs={"itemprop": "wordCount"})
        if len(novelWords) > 0:
            words = novelWords[0].text.replace('字', '')
            # 转成int
            words = int(words)
            if words == 0:
                print("字数为0，跳过\n")
                continue

        # ===================详细分析=============================
        end_time = start_time = ""
        # 最新更新时间+作者+书名（针对单章&多章作品）

        # <meta content="容若子安(作者),为谁等待（书名）,
        # 容若子安《为谁等待》之为谁等待,主角：宁若生洛夕 ┃ 配角： ┃ 其它：青春等待交错,
        # 最新更新:2014-12-26 20:36:17 作品积分：48426" name="Keywords"/>

        Keywords = soup.find_all('meta', attrs={"name": "Keywords"})
        # re.compile可以自定义正则表达式，获取多个匹配字段
        pattern_key = re.compile(r'content="(.*?),(.*?),.*?最新更新:(.*?)\s作品积分.*?"', re.S)
        novelInfos = pattern_key.findall(str(Keywords))
        if novelInfos:  # 如果匹配成功
            print("【小说名】:" + novelInfos[0][1])
            print("【作者】:" + novelInfos[0][0])
            print("【最新更新时间】：" + novelInfos[0][2])

            name = novelInfos[0][1]
            author = novelInfos[0][0]
            end_time = start_time = novelInfos[0][2]
        # 先把首发和最新更新都设为该时间，若是多章作品存在首发时间，再重新给首发时间赋值

        # ======================

        # 首发时间 (正则匹配)
        # <td align="center" title="章节存稿时间：2020-01-17 18:06:09 章节首发时间：2020-01-17 18:06:09">
        pattern = re.compile(r'章节首发时间：(.*?)">', re.S)
        novelTime = pattern.findall(res.text)
        if len(novelTime) > 0:
            start_time = novelTime[0]

        print("【首发时间】：" + start_time)

        print("【字数】:" + str(words))  # 补上字数

        # ======================
        # 小说类型
        novelType = soup.find_all('span', attrs={"itemprop": "genre"})
        if len(novelType) > 0:
            Type = novelType[0].text.strip()
            print("【小说类型】:" + Type)  # strip() 去除开头/结尾的空格，原本内容有好长空格。。

        # 小说自定义tag
        list1 = []
        _tags = ""
        novelTag = soup.find_all('a', attrs={"style": "text-decoration:none;color: red;"})
        if len(novelTag) > 0:
            for item in novelTag:  # 多个tag 拼接成一串，tag之间用'|'分割
                list1.append(item.text)  # 将内容提取出来组成一个列表
            _tags += '|'.join(list1)  # 用'|'分割
            print("【tag】:" + _tags)

        # =========小说成就=============
        # 点击    书评      收藏    礼物（营养液
        clicks = review = collect = gift = 0

        pattern_gain = re.compile(r'总点击数：\s*(.*?)\s*总书评数：(.*?)\s*当前被收藏数：(.*?)\s*文章积分：(.*?)\s*?</div>', re.S)
        novelGains = pattern_gain.findall(res.text)
        if novelGains:  # 小说成就（单篇幅作品）
            clicks = int(novelGains[0][0])  # 总点击数
            review = int(novelGains[0][1])  # 总书评数
            collect = int(novelGains[0][2])     # 总收藏数
            novelPoints = novelGains[0][3]      # 总积分  去除逗号
            novelPoints = novelPoints.replace(',', '')
            print("【点击】:" + novelGains[0][0] + "\n【书评】:" + novelGains[0][1] + "\n【收藏】:" + novelGains[0][2] + "\n【积分】：" + novelPoints)
            novelPoints = int(novelPoints)

        else:   # 小说成就（多章作品）
            novelScores = soup.find_all('div', attrs={"align": "center"})     # 一次扒的是一整块
            if len(novelScores) > 0:
                novelScores_text = str(novelScores)     # 转成字符串，切割

                # 书评
                novelReview = re.search('<span itemprop="reviewCount">(.*?)</span>', novelScores_text, re.S)
                if novelReview:  # 失败返回None
                    print("【书评】:" + novelReview.group(1))
                    # # 转成int
                    review = int(novelReview.group(1))
                else:
                    review = 0

                # 收藏
                novelCollect = re.search('<span itemprop="collectedCount">(.*?)</span>', novelScores_text, re.S)
                if novelCollect:
                    print("【收藏】:" + novelCollect.group(1))
                    # # 转成int
                    collect = int(novelCollect.group(1))
                else:
                    collect = 0

                # 营养液（礼物
                novelGift = re.search('<span>(.*?)</span>', novelScores_text, re.S)
                if novelGift:
                    print("【营养液】:" + novelGift.group(1))
                    # # 转成int
                    gift = int(novelGift.group(1))
                else:
                    gift = 0

                # 文章积分 \s*?可以过滤掉空白字符
                novelPoint = re.findall('文章积分：(.*?)\s*?</div>', novelScores_text, re.S)
                if novelPoint:
                    novelPoints = novelPoint[0]
                    # 去除空格
                    # novelPoints = novelPoints.strip()
                    # 去除逗号
                    novelPoints = novelPoints.replace(',', '')
                    print("【积分】：" + novelPoints)
                    # 转成int
                    novelPoints = int(novelPoints)
                else:
                    novelPoints = 0

        if gift == 0:  # 单篇幅无营养液
            a.do(
                f'INSERT INTO novel(novel.`name`,zishu, zuozhe, website,type,tag,startTime,endTime,dianji,yuanBookId,jReview,jFavorite,jPoint) VALUES("{name}","{words}","{author}","晋江","{Type}","{_tags}","{start_time}","{end_time}","{clicks}","{yuanBookId}","{review}","{collect}","{novelPoints}")')
        else:
            a.do(
                f'INSERT INTO novel(novel.`name`,zishu, zuozhe, website,type,tag,startTime,endTime,yuanBookId,jReview,jFavorite,jGifts,jPoint) VALUES("{name}","{words}","{author}","晋江","{Type}","{_tags}","{start_time}","{end_time}","{yuanBookId}","{review}","{collect}","{gift}","{novelPoints}")')

        print()
        time.sleep(1)
